/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package uit.ieproject.crawler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import java.util.ArrayList;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import jvntokenizer.PennTokenizer;
import uit.ieproject.presentation.MainFrame;

/**
 *
 * @author RedDevils
 */
public class NameCrawler extends WebCrawler {

    private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g" + "|png|tiff?|mid|mp2|mp3|mp4"
            + "|wav|avi|mov|mpeg|ram|m4v|pdf" + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
    private ArrayList<String> name = new ArrayList<>();

    public NameCrawler() {
    }

    @Override
    public boolean shouldVisit(WebURL url) {
        String href = url.getURL().toLowerCase();
        return !FILTERS.matcher(href).matches() && href.contains(MainFrame.URL);
    }

    @Override
    public void visit(Page page) {
        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            //String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();

            try {
                if (html.length() > 0) {
                    String str = PennTokenizer.parseHTML(html);
                    name = getArrStrWithPattern(Patterns.STR_CHARACTER, str);
                    if (name.size() > 0) {
                        for (int i = 0; i < name.size(); i++) {
                            if (name.get(i).length() > 0) {
                                String tit = PennTokenizer.tokenize(name.get(i));
                                String nameStandard = getStrWithPattern(Patterns.STR_NAME, tit);
                                if (nameStandard.length() > 0) {
                                    MainFrame.crawlerListModel.addElement(tit.trim());
                                    MainFrame.listNameChooser.ensureIndexIsVisible(i);
                                    //get link
                                    String link = getStrWithPatternGroup1(Patterns.STR_LINK, name.get(i));
                                    if (link != null) {
                                        MainFrame.map.put(tit.trim(), link);
                                    }
                                }
                            }
                        }
                    }
                }
            } catch (Exception ex) {
                Logger.getLogger(MyCrawler.class.getName()).log(Level.SEVERE, null, ex);
            }
        }

        System.out.println("=============");
    }

    public String getStrWithPattern(String regex, String inputStr) {
        String returnStr = "";
        Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
        Matcher matcher = pattern.matcher(inputStr);
        //get only first occurrence
        while (matcher.find()) {
            returnStr = matcher.group();
        }
        return returnStr;
    }

    public ArrayList<String> getArrStrWithPattern(String regex, String inputStr) {
        ArrayList<String> list = new ArrayList<>();
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(inputStr);

        //get only first occurrence
        while (matcher.find()) {
            String returnStr = matcher.group();
            //remove html tags
            list.add(returnStr);
        }
        return list;
    }

    public String getStrWithPatternGroup1(String regex, String inputStr) {
        String returnStr = "";
        Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
        Matcher matcher = pattern.matcher(inputStr);
        //get only first occurrence
        while (matcher.find()) {
            returnStr = matcher.group(1);
        }
        return returnStr;
    }
}
